import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.Iterator;
import java.util.Vector;

import javax.naming.spi.DirectoryManager;

import websphinx.CrawlEvent;
import websphinx.Link;


public class DataSetBuilder {

	private String fileName;

	public DataSetBuilder(String fileName) throws IOException{
		this.fileName=fileName;
		HandleUrls();
	}

	public DataSetBuilder(){

	}
	
	public void HandleUrls() throws IOException{
		String currUrl;
		FileReader fr = new FileReader(new File(this.fileName));
		BufferedReader br = new BufferedReader(fr);
		currUrl= br.readLine();
		while (currUrl!=null){
			CrawlUrl("http://" + currUrl);
			currUrl=br.readLine();
		}
	}

	public String BuildFileName(String str){
		//all urls in this list are of the form www.XXXX.YYY

		//find first index of '.'
		int first = str.indexOf('.')+1;
		//find second index of '.'
		int second = str.indexOf('.',first);

		if ((first>-1) && (second >-1))
			return (str.substring(first,second)+".txt");
		else if (second>-1)
			return (str.substring(0,second)+".txt");
		else
			return str;
	}

	public String BuildDirectoryName(String str){
		//all urls in this list are of the form www.XXXX.YYY

		//find first index of '.'
		int first = str.indexOf('.')+1;
		//find second index of '.'
		int second = str.indexOf('.',first);

		return (str.substring(first,second));
	}


	public void CrawlUrl(String url) throws IOException{
		String urlFileName = BuildFileName(url);
		//open file for writing
		BufferedWriter bw = new BufferedWriter(new FileWriter(urlFileName ));

		UrlHarvestingCrawler crawler = new UrlHarvestingCrawler(url) ;
		crawler.run();

		if (crawler.getState()==CrawlEvent.STOPPED){
			PrintCrawlerResultsToFile(crawler,bw);
			bw.close();
			System.out.println("Finished to crawl " + (new URL(url)).getHost()  
					+" . Found " + crawler.getWebLinks().size() + 
					" links. See results in " + urlFileName );
		}
	}

	public void PrintCrawlerResultsToFile(UrlHarvestingCrawler crawler,BufferedWriter bw) 
	throws IOException{
		Collection<ExtendedLink> links = crawler.getWebLinks().values();
		for (Iterator<ExtendedLink> iter = links.iterator(); iter.hasNext();){
			ExtendedLink current = iter.next();
			bw.write(current.getLink().getURL() + "\r\n");
		}
	}

	/*public MyCrawler SetCrawlerConfig(String fileName,String url) 
	throws IOException{

		FileReader fr = new FileReader(new File(fileName));
		BufferedReader br = new BufferedReader(fr);
		int maxDepth = 0, maxPageInQ=0,maxPageToCollect=0;
		String crawlerDomain = null;
		//read config file

		String configParam = br.readLine().trim();
		while (configParam!=null){
			if (configParam.equals("Max_Depth")){
				configParam = br.readLine();
				maxDepth = Integer.valueOf(configParam);
				configParam = br.readLine();
			}
			else if (configParam.equals("Max_Page_In_Q")){
				configParam = br.readLine();
				maxPageInQ = Integer.valueOf(configParam);
				configParam = br.readLine();
			}
			else if (configParam.equals("Max_Page_To_Collect")){
				configParam = br.readLine();
				maxPageToCollect = Integer.valueOf(configParam);
				configParam = br.readLine();
			}
			else if (configParam.equals("Domain")){
				configParam = br.readLine();
				crawlerDomain = configParam;
				configParam = br.readLine();
			}
			else
				System.out.println("There is an error in the crawler configuration file (" 
						+ configParam + ")");
		}
		return (new MyCrawler(maxDepth,maxPageInQ,maxPageToCollect,
				crawlerDomain,("http://"+url)));
	}*/



	public static void main(String[] args) {

		//open file that contains urls to crawl
		String fileName;
		if (args.length==0)
			fileName="sites.txt";
		else
			fileName=args[0];
		/*		try {
			DataSetBuilder dsb = new DataSetBuilder(fileName);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}*/

	}

}
